package com.smoed.common.utilities.html;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.math.stat.StatUtils;

import com.smoed.common.utilities.ArrayMath;
import com.smoed.common.utilities.StringUtils;

public final class HTMLContentExtractor {

    // Patterns of useless junk that can be removed from the html.
    // My attempts at combining these regular expressions significantly degraded
    // performance, so I'm leaving them separated out.
    private final Pattern[] removePatterns = {
            Pattern.compile("<noscript(?:[^<]++|<(?!/noscript>))*+</noscript>", Pattern.CASE_INSENSITIVE),
            Pattern.compile("<style(?:[^<]++|<(?!/style>))*+</style>", Pattern.CASE_INSENSITIVE),
            Pattern.compile("<script(?:[^<]++|<(?!/script>))*+</script>", Pattern.CASE_INSENSITIVE),
            Pattern.compile("<head(?:[^<]++|<(?!/head>))*+</head>", Pattern.CASE_INSENSITIVE),
            Pattern.compile("<!--[\\s\\S]*?-->", Pattern.CASE_INSENSITIVE), // html comments
            Pattern.compile("<![^>]+>", Pattern.CASE_INSENSITIVE), // DOCTYPE tags etc...
            Pattern.compile("all\\s+rights\\s+reserved", Pattern.CASE_INSENSITIVE), // Copyright statements
            Pattern.compile("copyright\\s*(&copy;)?\\s*\\d{4}", Pattern.CASE_INSENSITIVE), // Copyright statements
    };

    private final Pattern titlePattern = Pattern.compile("<title>(.*?)</title>", Pattern.CASE_INSENSITIVE);

    // Pattern to match html beginning or ending tags
    private final Pattern tagPattern = Pattern.compile("(<\\w+\\s*[^>]*>)|(</\\w+>)", Pattern.CASE_INSENSITIVE);

    // For collapsing tags on a page, prevents a single tag from being on multiple lines
    private final Pattern collapseTagsPattern = Pattern.compile("<[^>]+((\n)|(^>))*>", Pattern.CASE_INSENSITIVE);

    // For getting rid of strings like &nbsp;
    private final Pattern charCodePattern = Pattern.compile("&#*[\\w\\d]{0,6};", Pattern.CASE_INSENSITIVE);

    // For counting valid characters in the text-to-tag ratio
    private final Pattern validCharsPattern = Pattern.compile("[\\w:;'\".,?!#$%&()-_]", Pattern.CASE_INSENSITIVE);

    private final Pattern collapseWhitespacePattern = Pattern.compile("  +", Pattern.CASE_INSENSITIVE);

    private final Pattern wordOrSpacePattern = Pattern.compile("[\\w\\s]", Pattern.CASE_INSENSITIVE);

    private final String newLineDelimter = "\\s*[\r\n]+";

    // The radius of the smoothing filter's calculations
    private static final int SMOOTHING_RADIUS = 2;

    // Initial size of the StringBuffer used for the final content string
    private static final int INITIAL_CONTENT_BUFFER_SIZE = 1000;

    // Set an absolute minimum TTR threshold to filter out sites with little
    // content
    private static final double MIN_TTR_THRESHOLD = 10;

    // A high vertical bar to text ratio usually indicates navigation menus
    private static final double MAX_VERTICAL_BAR_RATIO = 0.01;

    // A low word to non word character ratio usually indicates little content
    private static final double MIN_WORD_TO_NON_WORD_RATIO = 0.90;

    public String extractTitle(String html) {
        final Matcher m = titlePattern.matcher(html);
        return (m.find()) ?  m.group(1) : "";
    }

    /*
     * Takes in raw html as a string and returns its best guess as to what
     * the relevant textual content is. Attempts to eliminate "junk" such as
     * navigation text, footers, and advertisements. Algorithm taken largely
     * from the paper "Text Extraction from the Web via Text-to-Tag Ratio"
     * published by Kansas State University.
     */
    public String extractContent(String html) {
        // Clean up junk inside html
        final String cleanHtml = cleanHtmlString(html);

        // Populate an array of text-to-tag ratios from the lines in the html
        final String[] lines = cleanHtml.split(newLineDelimter);

        if (lines.length == 0) {
            return "";
        }

        final double[] textToTagRatios = new double[lines.length];
        for (int i=0; i < lines.length; i++) {
            final String line = lines[i].trim();

            final int htmlTags = getNumberOfHtmlTags(line);

            // Remove html tags and surrounding whitespace from line
            final String cleanLine = StringUtils.replace(line, tagPattern, "");

            // Replace old line with clean line for later use
            lines[i] = cleanLine;

            // Count characters of line that are not part of html tags and
            // are not whitespace or other excluded characters
            final int nonTagCharacters = getNumberOfValidCharacters(cleanLine);

            // Calculate the text-to-tag ratio and add to array
            if (htmlTags == 0) {
                textToTagRatios[i] = nonTagCharacters;
            } else {
                textToTagRatios[i] = nonTagCharacters / (double) htmlTags;
            }

            // Log.info(this, "tags = " + htmlTags + "  ratio = " + textToTagRatios[i] + "  " + cleanLine);
        }

        // Smooth out the array of ratios so we include boundary content (such as content titles)
        final double[] smoothedRatios = ArrayMath.getSmoothedArraySimple(textToTagRatios, SMOOTHING_RADIUS);

        // Gather statistics so that we can define a threshold
        final double mean = StatUtils.mean(smoothedRatios);
        final double stdDeviation = Math.sqrt(StatUtils.variance(smoothedRatios, mean));

        // Only accept content lines scoring at least one standard deviation
        // above the mean
        double threshold = stdDeviation + mean;

        // Make sure the threshold meets the minimum threshold requirement
        threshold = threshold > MIN_TTR_THRESHOLD ? threshold : MIN_TTR_THRESHOLD;

        // Filter out lines not meeting the text-to-tag ratio, blank lines,
        // and those that don't meet certain other filtering criteria
        final StringBuffer content = new StringBuffer(INITIAL_CONTENT_BUFFER_SIZE);
        for (int i=0; i < smoothedRatios.length; i++) {
            // Log.info(this, smoothedRatios[i] + "  " + lines[i]);

            if (textToTagRatios[i] != 0 && smoothedRatios[i] > threshold && isLineValid(lines[i])) {
                content.append(lines[i].trim());
                content.append('\n');
            }
        }
        //Log.info(this, "Mean = " + mean + ", Std Dev = " + stdDeviation + ", Threshold = " + threshold);

        return StringUtils.replace(content.toString(), collapseWhitespacePattern, " ");
    }

    private boolean isLineValid(String line) {
        final int verticalBars = StringUtils.getNumCharOccurrences(line, '|');

        boolean valid = true;
        if (verticalBars / (double) line.length() > MAX_VERTICAL_BAR_RATIO) {
            // Large numbers of vertical bars usually indicate navigation menus
            valid = false;
        } else if (StringUtils.countMatches(line, wordOrSpacePattern)
                / (double) line.length() < MIN_WORD_TO_NON_WORD_RATIO) {
            // Large numbers of non word characters usually indicate junk
            valid = false;
        }

        return valid;
    }

    private String cleanHtmlString(String html) {
        String cleanHtml = html;
        for (Pattern pattern : removePatterns) {
            cleanHtml = StringUtils.replace(cleanHtml, pattern, "");
        }
        cleanHtml = StringUtils.replace(cleanHtml, collapseTagsPattern, "<a>");
        cleanHtml = StringUtils.replace(cleanHtml, charCodePattern, " ");

        return cleanHtml;
    }

    private int getNumberOfValidCharacters(String str) {
        return StringUtils.countMatches(str, validCharsPattern);
    }

    private int getNumberOfHtmlTags(String html) {
        return StringUtils.countMatches(html, tagPattern);
    }

}
